﻿using Microsoft.MachineLearning.CommandLine;
using Microsoft.TMSN.TMSNlearn;
using myAC.Configuration;
using myAC.DataProviders;
using myAC.FeatureVector;
using myAC.PreProcessing.DocLevel;
using myAC.sLDA;
using myAC.TLC;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Threading.Tasks;

namespace myAC
{
    class Program
    {
        static void Main(string[] args)
        {
            Config cfg = new Config();
            cfg.TextFeatureOnly = true;
            cfg.SRAmountThresholdForATopic = 50;  // If there're >50 srs for a category, we consider it trainable.
            cfg.WeightMethod = WeightMethod.COUNT; // COUNT is for sLDA http://www.cs.cmu.edu/~chongw/slda/readme.txt
            cfg.InputFileFormat = InputFileFormat.sLDA;
            Run2(cfg);
        }



        #region generalized
        static Func<myAC.DataProviders.ACInstance, SupportTopic> topicFunc = inst => (SupportTopic)(inst.OtherFeatures["Topic"]);

#if L1
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[0];
#elif L2
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[1];
#else
        static Func<myAC.DataProviders.ACInstance, String> callCodeFunc2 = inst => topicFunc(inst).CallCodeIdAtDifferentLevels[2];
#endif

        public static void Run2(Config cfg)
        {
            MSSolveSRProvider2 mss = new MSSolveSRProvider2();
            String rootCallCodes = "1001716,1001856,1002356"; //"1001856,1002356,1002243";
            Console.WriteLine("Getting SR data from DB...");            
            ACInstance[] insts = mss.GetData(rootCallCodes);
            Console.WriteLine();
            Console.WriteLine("Obtained {0} SRs in total.", insts.Length);
            if(insts.Length==0)
            {
                Console.WriteLine("Failed to get data...Quit.");
                return;
            }

            //The original callcode amount.
            Int32 originalCallCodeAmount = insts.GroupBy<ACInstance, String>(callCodeFunc2).Select(group => group.Key).Distinct().Count();

            //Keep only the callcodes with enough SRs.
            Console.WriteLine("Callcode SR amount threshold: {0}", cfg.SRAmountThresholdForATopic);
            String[] callCodesWithEnoughSRs = insts.GroupBy<ACInstance, String>(callCodeFunc2).Where(group => group.Count() > cfg.SRAmountThresholdForATopic).
                Select(group => group.Key).Distinct().ToArray();

            //Filter out the SRs belonging to the minority callcodes.
            ACInstance[] survivingInsts = insts.Where<ACInstance>(sr => callCodesWithEnoughSRs.Contains(callCodeFunc2(sr))).ToArray();

            //Callcode amount after filtering.
            Int32 filteredCallCodeAmount = survivingInsts.GroupBy<ACInstance, String>(callCodeFunc2).Select(group => group.Key).Distinct().Count();

            Console.WriteLine("Callcode before filtering: {0}\nCallcode after filtering: {1}\nCallcode surviving rate: {2}%",
                originalCallCodeAmount, filteredCallCodeAmount, 100.0 * filteredCallCodeAmount / originalCallCodeAmount);
            if(survivingInsts.Length==0)
            {
                Console.WriteLine("Training criteria not met. No instnaces survived...QUIT");
                return;
            }
            

            //Preprocessing text features of the surviving SRs at DOC LEVEL.
            Console.WriteLine("Preprocessing text features of the surviving SRs at DOC LEVEL.");
            Int32 survivingInstsAmount = survivingInsts.Length;
            Parallel.For(0,survivingInstsAmount, i =>
            {
                ACInstance inst = survivingInsts[i];
                DocLevelPipeline docPipeline = new DocLevelPipeline();
                inst.TextFeatures["Processed_Ngim_ExternalTitle"] = docPipeline.Process(inst.TextFeatures["Ngim_ExternalTitle"]);
                docPipeline.Reset(); // Reset the pipeline state for next preprocessing.
                inst.TextFeatures["Processed_Ngim_CustomerReportedSymptoms"] = docPipeline.Process(inst.TextFeatures["Ngim_CustomerReportedSymptoms"]);
            });

            //Generate doc-term vectors for TextFeatures. WORD LEVEL pre-processing happens here.
            Console.WriteLine("Generate doc-term vectors for TextFeatures. WORD LEVEL pre-processing happens here.");
            DTVHelper dtvHelper = new DTVHelper();                       
            String[] textFeatureFilter = { "Processed_Ngim_ExternalTitle", "Processed_Ngim_CustomerReportedSymptoms" }; //DTV is based on selected text features only.
            Dictionary<Int32, Int32>[] DTVs = dtvHelper.GenerateDTVs(survivingInsts, textFeatureFilter, cfg);
            //Further process of DTVs' weight as BIT or TFIDF.
            Console.WriteLine("Further process of DTVs' weight as {0}", cfg.WeightMethod.ToString());
            Dictionary<Int32, Int32>[] processedDTVs = dtvHelper.ProcessDTVs(DTVs, cfg);
            //Dictionary<Int32, Int32>[] processedDTVs = DTVs;


            //Generate input files based on instances' ALL features, not just text features.
            Console.WriteLine("Generate input files for selected algorithms: {0}.", cfg.InputFileFormat);
            String[] otherFeatureFilter = { "Topic" };

            String inputFileName = cfg.InputFileFormat.ToString() + ".txt";
            Int32 totalFeatureCount = 0;

            switch (cfg.InputFileFormat)
            {
                case InputFileFormat.sLDA:
                    totalFeatureCount = dtvHelper.term_id.Count; // sLDA accept TextFeatures only.
                    sLDAInputFileGenerator.GenerateInputData(survivingInsts, inputFileName, processedDTVs, otherFeatureFilter, totalFeatureCount, dtvHelper, cfg);
                    break;
                case InputFileFormat.TLC:
                default:
                    totalFeatureCount = dtvHelper.term_id.Count + otherFeatureFilter.Length; // For general classification methods, we can use not just the terms from TextFeatures, but also the OtherFeatures.
                    tlcInputFileGenerator.GenerateInputData(survivingInsts, inputFileName, processedDTVs, otherFeatureFilter, totalFeatureCount, dtvHelper, cfg);
                    break;
            }
            //Invoke TLC           

        }
        #endregion
        
        #region not-generalized
//#if L1
//        static Func<SR, String> callCodeFunc = sr => sr.Topic.CallCodeIdAtDifferentLevels[0];
//#elif L2
//        static Func<SR, String> callCodeFunc = sr => sr.Topic.CallCodeIdAtDifferentLevels[1];
//#elif L3
//        static Func<SR, String> callCodeFunc = sr => sr.Topic.CallCodeIdAtDifferentLevels[2];
//#endif

//        public static void Run(Config config)
//        {
//            MSSolveSRProvider mss = new MSSolveSRProvider();
//            String rootCallCodes = "1001856,1002356,1002243";
//            List<SR> srs = mss.GetData(rootCallCodes);

//            Console.WriteLine();
//            Console.WriteLine("Loaded {0} SRs.", srs.Count);

//            //The original callcode amount.
//            Int32 originalCallCodeAmount = srs.GroupBy<SR, String>(callCodeFunc).Select(group => group.Key).Distinct().Count();

//            //Keep only the callcodes with enough SRs.
//            String[] callCodesWithEnoughSRs = srs.GroupBy<SR, String>(callCodeFunc).Where(group => group.Count() > config.SRAmountThresholdForATopic).
//                Select(group => group.Key).ToArray();

//            //Filter out the SRs belonging to the minority callcodes.
//            List<SR> srs2 = srs.Where<SR>(sr => callCodesWithEnoughSRs.Contains(callCodeFunc(sr))).ToList();

//            //Callcode amount after filtering.
//            Int32 filteredCallCodeAmount = srs2.GroupBy<SR, String>(callCodeFunc).Select(group => group.Key).Distinct().Count();

//            Console.WriteLine("Callcode before filtering: {0}\nCallcode after filtering: {1}\nCallcode surviving rate: {2}%",
//                originalCallCodeAmount, filteredCallCodeAmount, 100.0 * filteredCallCodeAmount / originalCallCodeAmount);

//            //Preprocessing: we need to process all necessary features of a SR, including the applicable ones in OtherFeatures dictionary.


//            //Generate feature vectors based on SRs' all features.


//            //Generate TLC input files


//            //Invoke TLC

//        }
        #endregion
    }
}
